#!/usr/bin/env python
# import re
from scrapy import Request, Spider
from bots.spiderbot.spiderbot.items import JDProductItem


class JDProductSpider(Spider):
    name = 'jd_product'
    page_num = 5
    keyword_list = ['轮胎', '电瓶', '雨刮片', '刹车片', '机油', '火花塞', '玻璃水', '防冻液',
                    '节气门清洗剂', '车灯']
    base_url = 'https://search.jd.com/Search?keyword=%s&enc=utf-8&page=%d'

    m_name = ''
    m_brand = ''
    m_price = ''
    m_description = ''
    m_attribute = ''

    def start_requests(self):
        for keyword in self.keyword_list:
            for page in range(1, self.page_num, 2):
                url = self.base_url % (keyword, page)
                yield Request(url=url, callback=self.parse, meta={'keyword': keyword, 'middleware': True})

    def parse(self, response):
        keyword = response.meta['keyword']
        print('Begin parse', response.url)
        products = response.xpath('.//ul[@class="gl-warp clearfix"]/li')

        for product in products:
            price = ''.join(product.xpath('.//div[@class="p-price"]//i/text()').extract()).strip()
            sku = ''.join(product.xpath('./@data-sku').extract()).strip()
            url = "https://item.jd.com/" + sku + ".html"
            yield Request(url=url, callback=self.parse_subpage,
                          meta={'keyword': keyword, 'price': price, 'middleware': False})

    def parse_subpage(self, response):
        self.m_name = response.meta['keyword']
        self.m_brand = ''.join(response.xpath('.//ul[@id="parameter-brand"]/li/@title').extract_first()).strip()
        self.m_price = response.meta['price']
        self.m_description = ''.join(response.xpath('.//div[@class="sku-name"]/text()').extract()).strip()
        parameter_list = response.xpath('.//ul[@class="parameter2 p-parameter-list"]/li/text()').extract()
        tmp = ''

        for parameter in parameter_list:
            parameter_str = ''.join(parameter).strip()
            tmp += parameter_str

        self.m_attribute = tmp

        item = JDProductItem()
        item['product_name'] = self.m_name
        item['product_brand'] = self.m_brand
        item['product_price'] = self.m_price
        item['product_description'] = self.m_description
        item['product_attribute'] = self.m_attribute
        yield item
        print(item)
